{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/viniciusoliveirasd/bert-applications/blob/master/FakeNews.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zpujtPOVsUFh"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import zipfile\n",
    "import datetime\n",
    "import json\n",
    "import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aJZRUh_Zspaf"
   },
   "outputs": [],
   "source": [
    "VOCAB = 'bert-base-chinese'\n",
    "MODEL = 'bert-base-chinese'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from pytorch_pretrained_bert import BertTokenizer, BertModel, BertForMaskedLM, BertForSequenceClassification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained(VOCAB)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[CLS]',\n",
       " 'who',\n",
       " 'was',\n",
       " 'jim',\n",
       " 'he',\n",
       " '##ns',\n",
       " '##on',\n",
       " '?',\n",
       " '[SEP]',\n",
       " 'jim',\n",
       " 'he',\n",
       " '##ns',\n",
       " '##on',\n",
       " 'was',\n",
       " 'a',\n",
       " 'pu',\n",
       " '##pp',\n",
       " '##et',\n",
       " '##ee',\n",
       " '##r',\n",
       " '[SEP]']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = \"[CLS] Who was Jim Henson ? [SEP] Jim Henson was a puppeteer [SEP]\"\n",
    "tokenized_text = tokenizer.tokenize(text)\n",
    "tokenized_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['su']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.tokenize('su')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_CSV_PATH = './input/train.csv'\n",
    "TEST_CSV_PATH = './input/test.csv'\n",
    "TOKENIZED_TRAIN_CSV_PATH = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(TRAIN_CSV_PATH, index_col='id')\n",
    "test = pd.read_csv(TEST_CSV_PATH, index_col='id')\n",
    "cols = ['title1_zh', \n",
    "        'title2_zh', \n",
    "        'label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/gabrielbarcik/miniconda3/envs/deep_learning/lib/python3.7/site-packages/pandas/core/indexing.py:1494: FutureWarning: \n",
      "Passing list-likes to .loc or [] with any missing label will raise\n",
      "KeyError in the future, you can use .reindex() as an alternative.\n",
      "\n",
      "See the documentation here:\n",
      "https://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate-loc-reindex-listlike\n",
      "  return self._getitem_tuple(key)\n"
     ]
    }
   ],
   "source": [
    "train = train.loc[:, cols]\n",
    "test = test.loc[:, cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title1_zh</th>\n",
       "      <th>title2_zh</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2017养老保险又新增两项，农村老人人人可申领，你领到了吗</td>\n",
       "      <td>警方辟谣“鸟巢大会每人领5万” 仍有老人坚持进京</td>\n",
       "      <td>unrelated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>\"你不来深圳，早晚你儿子也要来\"，不出10年深圳人均GDP将超香港</td>\n",
       "      <td>深圳GDP首超香港？深圳统计局辟谣：只是差距在缩小</td>\n",
       "      <td>unrelated</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\"你不来深圳，早晚你儿子也要来\"，不出10年深圳人均GDP将超香港</td>\n",
       "      <td>GDP首超香港？深圳澄清：还差一点点……</td>\n",
       "      <td>unrelated</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            title1_zh                  title2_zh      label\n",
       "id                                                                         \n",
       "0       2017养老保险又新增两项，农村老人人人可申领，你领到了吗   警方辟谣“鸟巢大会每人领5万” 仍有老人坚持进京  unrelated\n",
       "3   \"你不来深圳，早晚你儿子也要来\"，不出10年深圳人均GDP将超香港  深圳GDP首超香港？深圳统计局辟谣：只是差距在缩小  unrelated\n",
       "1   \"你不来深圳，早晚你儿子也要来\"，不出10年深圳人均GDP将超香港       GDP首超香港？深圳澄清：还差一点点……  unrelated"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.fillna('UNKNOWN', inplace=True)\n",
    "test.fillna('UNKNOWN', inplace=True)\n",
    "train.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'unrelated': 219313, 'agreed': 92973, 'disagreed': 8266})"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from collections import Counter\n",
    "Counter(train.label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2019-03-13 20:09:01--  https://raw.githubusercontent.com/huggingface/pytorch-pretrained-BERT/master/examples/run_classifier.py\n",
      "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.16.133\n",
      "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.16.133|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 28675 (28K) [text/plain]\n",
      "Saving to: ‘run_classifier.py.2’\n",
      "\n",
      "run_classifier.py.2 100%[===================>]  28.00K  --.-KB/s    in 0.01s   \n",
      "\n",
      "2019-03-13 20:09:01 (2.68 MB/s) - ‘run_classifier.py.2’ saved [28675/28675]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget https://raw.githubusercontent.com/huggingface/pytorch-pretrained-BERT/master/examples/run_classifier.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "VALIDATION_RATIO = 0.1\n",
    "\n",
    "RANDOM_STATE = 9527\n",
    "\n",
    "train, val =  train_test_split(train, test_size=VALIDATION_RATIO, random_state=RANDOM_STATE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_list = ['unrelated', 'agreed', 'disagreed']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "288496"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from run_classifier import *\n",
    "\n",
    "train_examples = [InputExample('train', row.title1_zh, row.title2_zh, row.label) for row in train.itertuples()]\n",
    "val_examples = [InputExample('val', row.title1_zh, row.title2_zh, row.label) for row in val.itertuples()]\n",
    "test_examples = [InputExample('test', row.title1_zh, row.title2_zh, 'unrelated') for row in test.itertuples()]\n",
    "\n",
    "len(train_examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "orginal_total = len(train_examples)\n",
    "train_examples = train_examples[:int(orginal_total*0.2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "n_gpu = torch.cuda.device_count()\n",
    "gradient_accumulation_steps = 1\n",
    "train_batch_size = 32\n",
    "eval_batch_size = 128\n",
    "train_batch_size = train_batch_size // gradient_accumulation_steps\n",
    "output_dir = 'output'\n",
    "bert_model = 'bert-base-chinese'\n",
    "num_train_epochs = 3\n",
    "num_train_optimization_steps = int(\n",
    "            len(train_examples) / train_batch_size / gradient_accumulation_steps) * num_train_epochs\n",
    "cache_dir = \"model\"\n",
    "learning_rate = 5e-5\n",
    "warmup_proportion = 0.1\n",
    "max_seq_length = 128\n",
    "label_list = ['unrelated', 'agreed', 'disagreed']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/13/2019 20:09:14 - INFO - pytorch_pretrained_bert.tokenization -   loading vocabulary file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-chinese-vocab.txt from cache at /home/gabrielbarcik/.pytorch_pretrained_bert/8a0c070123c1f794c42a29c6904beb7c1b8715741e235bee04aca2c7636fc83f.9b42061518a39ca00b8b52059fd2bede8daa613f8a8671500e518a8c29de8c00\n"
     ]
    }
   ],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained(VOCAB)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/13/2019 20:10:48 - INFO - pytorch_pretrained_bert.modeling -   loading archive file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-chinese.tar.gz from cache at model/42d4a64dda3243ffeca7ec268d5544122e67d9d06b971608796b483925716512.02ac7d664cff08d793eb00d6aac1d04368a1322435e5fe0a27c70b0b3a85327f\n",
      "03/13/2019 20:10:48 - INFO - pytorch_pretrained_bert.modeling -   extracting archive file model/42d4a64dda3243ffeca7ec268d5544122e67d9d06b971608796b483925716512.02ac7d664cff08d793eb00d6aac1d04368a1322435e5fe0a27c70b0b3a85327f to temp dir /tmp/tmp_yrpg0mi\n",
      "03/13/2019 20:10:51 - INFO - pytorch_pretrained_bert.modeling -   Model config {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"directionality\": \"bidi\",\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pooler_fc_size\": 768,\n",
      "  \"pooler_num_attention_heads\": 12,\n",
      "  \"pooler_num_fc_layers\": 3,\n",
      "  \"pooler_size_per_head\": 128,\n",
      "  \"pooler_type\": \"first_token_transform\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"vocab_size\": 21128\n",
      "}\n",
      "\n",
      "03/13/2019 20:10:55 - INFO - pytorch_pretrained_bert.modeling -   Weights of BertForSequenceClassification not initialized from pretrained model: ['classifier.weight', 'classifier.bias']\n",
      "03/13/2019 20:10:55 - INFO - pytorch_pretrained_bert.modeling -   Weights from pretrained model not used in BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(BertForSequenceClassification(\n",
       "   (bert): BertModel(\n",
       "     (embeddings): BertEmbeddings(\n",
       "       (word_embeddings): Embedding(21128, 768)\n",
       "       (position_embeddings): Embedding(512, 768)\n",
       "       (token_type_embeddings): Embedding(2, 768)\n",
       "       (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "       (dropout): Dropout(p=0.1)\n",
       "     )\n",
       "     (encoder): BertEncoder(\n",
       "       (layer): ModuleList(\n",
       "         (0): BertLayer(\n",
       "           (attention): BertAttention(\n",
       "             (self): BertSelfAttention(\n",
       "               (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (dropout): Dropout(p=0.1)\n",
       "             )\n",
       "             (output): BertSelfOutput(\n",
       "               (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "               (dropout): Dropout(p=0.1)\n",
       "             )\n",
       "           )\n",
       "           (intermediate): BertIntermediate(\n",
       "             (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "           )\n",
       "           (output): BertOutput(\n",
       "             (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "             (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "             (dropout): Dropout(p=0.1)\n",
       "           )\n",
       "         )\n",
       "         (1): BertLayer(\n",
       "           (attention): BertAttention(\n",
       "             (self): BertSelfAttention(\n",
       "               (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (dropout): Dropout(p=0.1)\n",
       "             )\n",
       "             (output): BertSelfOutput(\n",
       "               (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "               (dropout): Dropout(p=0.1)\n",
       "             )\n",
       "           )\n",
       "           (intermediate): BertIntermediate(\n",
       "             (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "           )\n",
       "           (output): BertOutput(\n",
       "             (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "             (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "             (dropout): Dropout(p=0.1)\n",
       "           )\n",
       "         )\n",
       "         (2): BertLayer(\n",
       "           (attention): BertAttention(\n",
       "             (self): BertSelfAttention(\n",
       "               (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (dropout): Dropout(p=0.1)\n",
       "             )\n",
       "             (output): BertSelfOutput(\n",
       "               (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "               (dropout): Dropout(p=0.1)\n",
       "             )\n",
       "           )\n",
       "           (intermediate): BertIntermediate(\n",
       "             (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "           )\n",
       "           (output): BertOutput(\n",
       "             (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "             (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "             (dropout): Dropout(p=0.1)\n",
       "           )\n",
       "         )\n",
       "         (3): BertLayer(\n",
       "           (attention): BertAttention(\n",
       "             (self): BertSelfAttention(\n",
       "               (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (dropout): Dropout(p=0.1)\n",
       "             )\n",
       "             (output): BertSelfOutput(\n",
       "               (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "               (dropout): Dropout(p=0.1)\n",
       "             )\n",
       "           )\n",
       "           (intermediate): BertIntermediate(\n",
       "             (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "           )\n",
       "           (output): BertOutput(\n",
       "             (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "             (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "             (dropout): Dropout(p=0.1)\n",
       "           )\n",
       "         )\n",
       "         (4): BertLayer(\n",
       "           (attention): BertAttention(\n",
       "             (self): BertSelfAttention(\n",
       "               (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (dropout): Dropout(p=0.1)\n",
       "             )\n",
       "             (output): BertSelfOutput(\n",
       "               (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "               (dropout): Dropout(p=0.1)\n",
       "             )\n",
       "           )\n",
       "           (intermediate): BertIntermediate(\n",
       "             (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "           )\n",
       "           (output): BertOutput(\n",
       "             (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "             (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "             (dropout): Dropout(p=0.1)\n",
       "           )\n",
       "         )\n",
       "         (5): BertLayer(\n",
       "           (attention): BertAttention(\n",
       "             (self): BertSelfAttention(\n",
       "               (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (dropout): Dropout(p=0.1)\n",
       "             )\n",
       "             (output): BertSelfOutput(\n",
       "               (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "               (dropout): Dropout(p=0.1)\n",
       "             )\n",
       "           )\n",
       "           (intermediate): BertIntermediate(\n",
       "             (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "           )\n",
       "           (output): BertOutput(\n",
       "             (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "             (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "             (dropout): Dropout(p=0.1)\n",
       "           )\n",
       "         )\n",
       "         (6): BertLayer(\n",
       "           (attention): BertAttention(\n",
       "             (self): BertSelfAttention(\n",
       "               (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (dropout): Dropout(p=0.1)\n",
       "             )\n",
       "             (output): BertSelfOutput(\n",
       "               (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "               (dropout): Dropout(p=0.1)\n",
       "             )\n",
       "           )\n",
       "           (intermediate): BertIntermediate(\n",
       "             (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "           )\n",
       "           (output): BertOutput(\n",
       "             (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "             (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "             (dropout): Dropout(p=0.1)\n",
       "           )\n",
       "         )\n",
       "         (7): BertLayer(\n",
       "           (attention): BertAttention(\n",
       "             (self): BertSelfAttention(\n",
       "               (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (dropout): Dropout(p=0.1)\n",
       "             )\n",
       "             (output): BertSelfOutput(\n",
       "               (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "               (dropout): Dropout(p=0.1)\n",
       "             )\n",
       "           )\n",
       "           (intermediate): BertIntermediate(\n",
       "             (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "           )\n",
       "           (output): BertOutput(\n",
       "             (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "             (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "             (dropout): Dropout(p=0.1)\n",
       "           )\n",
       "         )\n",
       "         (8): BertLayer(\n",
       "           (attention): BertAttention(\n",
       "             (self): BertSelfAttention(\n",
       "               (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (dropout): Dropout(p=0.1)\n",
       "             )\n",
       "             (output): BertSelfOutput(\n",
       "               (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "               (dropout): Dropout(p=0.1)\n",
       "             )\n",
       "           )\n",
       "           (intermediate): BertIntermediate(\n",
       "             (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "           )\n",
       "           (output): BertOutput(\n",
       "             (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "             (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "             (dropout): Dropout(p=0.1)\n",
       "           )\n",
       "         )\n",
       "         (9): BertLayer(\n",
       "           (attention): BertAttention(\n",
       "             (self): BertSelfAttention(\n",
       "               (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (dropout): Dropout(p=0.1)\n",
       "             )\n",
       "             (output): BertSelfOutput(\n",
       "               (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "               (dropout): Dropout(p=0.1)\n",
       "             )\n",
       "           )\n",
       "           (intermediate): BertIntermediate(\n",
       "             (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "           )\n",
       "           (output): BertOutput(\n",
       "             (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "             (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "             (dropout): Dropout(p=0.1)\n",
       "           )\n",
       "         )\n",
       "         (10): BertLayer(\n",
       "           (attention): BertAttention(\n",
       "             (self): BertSelfAttention(\n",
       "               (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (dropout): Dropout(p=0.1)\n",
       "             )\n",
       "             (output): BertSelfOutput(\n",
       "               (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "               (dropout): Dropout(p=0.1)\n",
       "             )\n",
       "           )\n",
       "           (intermediate): BertIntermediate(\n",
       "             (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "           )\n",
       "           (output): BertOutput(\n",
       "             (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "             (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "             (dropout): Dropout(p=0.1)\n",
       "           )\n",
       "         )\n",
       "         (11): BertLayer(\n",
       "           (attention): BertAttention(\n",
       "             (self): BertSelfAttention(\n",
       "               (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (dropout): Dropout(p=0.1)\n",
       "             )\n",
       "             (output): BertSelfOutput(\n",
       "               (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "               (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "               (dropout): Dropout(p=0.1)\n",
       "             )\n",
       "           )\n",
       "           (intermediate): BertIntermediate(\n",
       "             (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "           )\n",
       "           (output): BertOutput(\n",
       "             (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "             (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "             (dropout): Dropout(p=0.1)\n",
       "           )\n",
       "         )\n",
       "       )\n",
       "     )\n",
       "     (pooler): BertPooler(\n",
       "       (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "       (activation): Tanh()\n",
       "     )\n",
       "   )\n",
       "   (dropout): Dropout(p=0.1)\n",
       "   (classifier): Linear(in_features=768, out_features=3, bias=True)\n",
       " ), <pytorch_pretrained_bert.tokenization.BertTokenizer at 0x7f5c6613cf98>)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = BertForSequenceClassification.from_pretrained(MODEL,\n",
    "              cache_dir=cache_dir,\n",
    "              num_labels = 3)\n",
    "model.to(device)\n",
    "if n_gpu > 1:\n",
    "    model = torch.nn.DataParallel(model)\n",
    "model, tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_optimizer = list(model.named_parameters())\n",
    "no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n",
    "optimizer_grouped_parameters = [\n",
    "    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},\n",
    "    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n",
    "    ]\n",
    "optimizer = BertAdam(optimizer_grouped_parameters,\n",
    "                             lr=learning_rate,\n",
    "                             warmup=warmup_proportion,\n",
    "                             t_total=num_train_optimization_steps)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "03/13/2019 20:11:11 - INFO - run_classifier -   *** Example ***\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   guid: train\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   tokens: [CLS] 营 养 师 ： 补 充 这 4 种 营 养 能 帮 你 降 血 压 ， 你 一 样 都 不 吃 么 ？ [SEP] 刘 涛 担 心 前 夫 离 婚 后 找 不 到 另 一 半 ， 居 然 还 主 动 给 他 介 绍 女 人 [SEP]\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   input_ids: 101 5852 1075 2360 8038 6133 1041 6821 125 4905 5852 1075 5543 2376 872 7360 6117 1327 8024 872 671 3416 6963 679 1391 720 8043 102 1155 3875 2857 2552 1184 1923 4895 2042 1400 2823 679 1168 1369 671 1288 8024 2233 4197 6820 712 1220 5314 800 792 5305 1957 782 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   label: unrelated (id = 0)\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   *** Example ***\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   guid: train\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   tokens: [CLS] 刘 涛 现 场 痛 哭 发 飙 要 离 婚 ， 直 接 把 旁 边 的 沈 腾 吓 蒙 了 [SEP] 她 长 相 酷 似 刘 涛 ， 是 陈 思 诚 前 女 友 ， 离 婚 后 的 她 活 出 了 真 正 的 自 己 ！ [SEP]\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   input_ids: 101 1155 3875 4385 1767 4578 1526 1355 7604 6206 4895 2042 8024 4684 2970 2828 3178 6804 4638 3755 5596 1405 5885 749 102 1961 7270 4685 6999 849 1155 3875 8024 3221 7357 2590 6411 1184 1957 1351 8024 4895 2042 1400 4638 1961 3833 1139 749 4696 3633 4638 5632 2346 8013 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   label: unrelated (id = 0)\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   *** Example ***\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   guid: train\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   tokens: [CLS] nba 被 兜 售 球 星 诞 生 ！ 1 亿 美 金 篮 板 王 ！ 火 箭 骑 士 同 抢 交 易 ？ [SEP] 猛 龙 总 裁 辟 谣 球 队 重 建 传 闻 ， 主 帅 的 去 留 需 综 合 多 方 面 考 虑 [SEP]\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   input_ids: 101 8391 6158 1055 1545 4413 3215 6414 4495 8013 122 783 5401 7032 5074 3352 4374 8013 4125 5055 7744 1894 1398 2843 769 3211 8043 102 4338 7987 2600 6161 6792 6469 4413 7339 7028 2456 837 7319 8024 712 2358 4638 1343 4522 7444 5341 1394 1914 3175 7481 5440 5991 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   label: unrelated (id = 0)\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   *** Example ***\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   guid: train\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   tokens: [CLS] 海 口 飞 机 撒 药 治 白 蛾 [SEP] [UNK] 配 方 [UNK] [UNK] 味 道 [UNK] 没 变 ！ 飞 机 撒 药 治 白 蛾 谣 言 [SEP]\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   input_ids: 101 3862 1366 7607 3322 3054 5790 3780 4635 6042 102 100 6981 3175 100 100 1456 6887 100 3766 1359 8013 7607 3322 3054 5790 3780 4635 6042 6469 6241 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   label: unrelated (id = 0)\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   *** Example ***\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   guid: train\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   tokens: [CLS] 网 曝 杜 海 涛 与 沈 梦 辰 已 分 手 疑 似 女 方 劈 腿 [SEP] 爆 ： 杜 海 涛 与 沈 梦 辰 已 分 手 疑 似 女 方 出 轨 [SEP]\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   input_ids: 101 5381 3284 3336 3862 3875 680 3755 3457 6801 2347 1146 2797 4542 849 1957 3175 1207 5597 102 4255 8038 3336 3862 3875 680 3755 3457 6801 2347 1146 2797 4542 849 1957 3175 1139 6758 102 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   input_mask: 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   segment_ids: 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
      "03/13/2019 20:11:11 - INFO - run_classifier -   label: agreed (id = 1)\n",
      "03/13/2019 20:11:30 - INFO - run_classifier -   ***** Running training *****\n",
      "03/13/2019 20:11:30 - INFO - run_classifier -     Num examples = 57699\n",
      "03/13/2019 20:11:30 - INFO - run_classifier -     Batch size = 32\n",
      "03/13/2019 20:11:30 - INFO - run_classifier -     Num steps = 5409\n"
     ]
    }
   ],
   "source": [
    "global_step = 0\n",
    "nb_tr_steps = 0\n",
    "tr_loss = 0\n",
    "\n",
    "train_features = convert_examples_to_features(\n",
    "    train_examples, label_list, max_seq_length, tokenizer)\n",
    "logger.info(\"***** Running training *****\")\n",
    "logger.info(\"  Num examples = %d\", len(train_examples))\n",
    "logger.info(\"  Batch size = %d\", train_batch_size)\n",
    "logger.info(\"  Num steps = %d\", num_train_optimization_steps)\n",
    "all_input_ids = torch.tensor([f.input_ids for f in train_features], dtype=torch.long)\n",
    "all_input_mask = torch.tensor([f.input_mask for f in train_features], dtype=torch.long)\n",
    "all_segment_ids = torch.tensor([f.segment_ids for f in train_features], dtype=torch.long)\n",
    "all_label_ids = torch.tensor([f.label_id for f in train_features], dtype=torch.long)\n",
    "train_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids, all_label_ids)\n",
    "train_sampler = RandomSampler(train_data)\n",
    "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=train_batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### model.train()\n",
    "for _ in trange(int(num_train_epochs), desc=\"Epoch\"):\n",
    "    tr_loss = 0\n",
    "    nb_tr_examples, nb_tr_steps = 0, 0\n",
    "    total_step = len(train_data) // train_batch_size\n",
    "    ten_percent_step = total_step // 10\n",
    "    for step, batch in enumerate(train_dataloader):\n",
    "        batch = tuple(t.to(device) for t in batch)\n",
    "        input_ids, input_mask, segment_ids, label_ids = batch\n",
    "        loss = model(input_ids, segment_ids, input_mask, label_ids)\n",
    "        if n_gpu > 1:\n",
    "            loss = loss.mean() # mean() to average on multi-gpu.\n",
    "        if gradient_accumulation_steps > 1:\n",
    "            loss = loss / gradient_accumulation_steps\n",
    "\n",
    "        loss.backward()\n",
    "\n",
    "        tr_loss += loss.item()\n",
    "        nb_tr_examples += input_ids.size(0)\n",
    "        nb_tr_steps += 1\n",
    "        if (step + 1) % gradient_accumulation_steps == 0:\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad()\n",
    "            global_step += 1\n",
    "            \n",
    "        if step % ten_percent_step == 0:\n",
    "            print(\"Fininshed: {:.2f}% ({}/{})\".format(step/total_step*100, step, total_step))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "include_colab_link": true,
   "name": "FakeNews.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "deep_learning",
   "language": "python",
   "name": "deep_learning"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
